{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pandas==0.23.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (0.23.0)\n",
      "Requirement already satisfied: python-dateutil>=2.5.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pandas==0.23.0) (2.7.3)\n",
      "Requirement already satisfied: numpy>=1.9.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pandas==0.23.0) (1.14.3)\n",
      "Requirement already satisfied: pytz>=2011k in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pandas==0.23.0) (2018.4)\n",
      "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from python-dateutil>=2.5.0->pandas==0.23.0) (1.11.0)\n",
      "Requirement already satisfied: numpy==1.14.3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (1.14.3)\n",
      "Requirement already satisfied: matplotlib==3.0.3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (3.0.3)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib==3.0.3) (2.7.3)\n",
      "Requirement already satisfied: numpy>=1.10.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib==3.0.3) (1.14.3)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib==3.0.3) (2.2.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib==3.0.3) (1.0.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib==3.0.3) (0.10.0)\n",
      "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from python-dateutil>=2.1->matplotlib==3.0.3) (1.11.0)\n",
      "Requirement already satisfied: setuptools in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from kiwisolver>=1.0.1->matplotlib==3.0.3) (39.1.0)\n",
      "Requirement already satisfied: seaborn==0.8.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (0.8.1)\n",
      "Requirement already satisfied: numpy in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from seaborn==0.8.1) (1.14.3)\n",
      "Requirement already satisfied: scipy in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from seaborn==0.8.1) (1.1.0)\n",
      "Requirement already satisfied: matplotlib in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from seaborn==0.8.1) (3.0.3)\n",
      "Requirement already satisfied: pandas in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from seaborn==0.8.1) (0.23.0)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib->seaborn==0.8.1) (2.2.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib->seaborn==0.8.1) (1.0.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib->seaborn==0.8.1) (0.10.0)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib->seaborn==0.8.1) (2.7.3)\n",
      "Requirement already satisfied: pytz>=2011k in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pandas->seaborn==0.8.1) (2018.4)\n",
      "Requirement already satisfied: setuptools in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from kiwisolver>=1.0.1->matplotlib->seaborn==0.8.1) (39.1.0)\n",
      "Requirement already satisfied: six in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from cycler>=0.10->matplotlib->seaborn==0.8.1) (1.11.0)\n",
      "Requirement already satisfied: PyAthena==1.8.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (1.8.0)\n",
      "Requirement already satisfied: boto3>=1.4.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from PyAthena==1.8.0) (1.11.13)\n",
      "Requirement already satisfied: tenacity>=4.1.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from PyAthena==1.8.0) (6.0.0)\n",
      "Requirement already satisfied: future in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from PyAthena==1.8.0) (0.18.2)\n",
      "Requirement already satisfied: botocore>=1.5.52 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from PyAthena==1.8.0) (1.14.13)\n",
      "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from boto3>=1.4.4->PyAthena==1.8.0) (0.3.3)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from boto3>=1.4.4->PyAthena==1.8.0) (0.9.4)\n",
      "Requirement already satisfied: six>=1.9.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from tenacity>=4.1.0->PyAthena==1.8.0) (1.11.0)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from botocore>=1.5.52->PyAthena==1.8.0) (2.7.3)\n",
      "Requirement already satisfied: urllib3<1.26,>=1.20; python_version != \"3.4\" in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from botocore>=1.5.52->PyAthena==1.8.0) (1.23)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from botocore>=1.5.52->PyAthena==1.8.0) (0.14)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using pip version 10.0.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\n",
      "You are using pip version 10.0.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\n",
      "You are using pip version 10.0.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\n",
      "You are using pip version 10.0.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\n",
      "You are using pip version 10.0.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "\n",
    "pip install pandas==0.23.0\n",
    "pip install numpy==1.14.3\n",
    "pip install matplotlib==3.0.3\n",
    "pip install seaborn==0.8.1\n",
    "pip install PyAthena==1.8.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "from scipy.sparse import lil_matrix\n",
    "\n",
    "import boto3\n",
    "import botocore\n",
    "import sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "session = boto3.session.Session()\n",
    "region_name = session.region_name\n",
    "\n",
    "sagemaker_session = sagemaker.Session()\n",
    "bucket = sagemaker_session.default_bucket()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyathena import connect\n",
    "from pyathena.util import as_pandas\n",
    "\n",
    "database_name = 'dsoaws'\n",
    "table_name = 'amazon_reviews_parquet'\n",
    "sentiment_table_name = 'amazon_reviews_with_sentiment'\n",
    "\n",
    "s3_staging_dir = 's3://{0}/staging/athena'.format(bucket)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "cursor = connect(region_name=region_name, s3_staging_dir=s3_staging_dir).cursor()\n",
    "\n",
    "cursor.execute('SELECT customer_id, \\\n",
    "                         review_id, \\\n",
    "                         product_id, \\\n",
    "                         product_title, \\\n",
    "                         review_headline, \\\n",
    "                         review_body, \\\n",
    "                         review_date, \\\n",
    "                         year, \\\n",
    "                         star_rating, \\\n",
    "                         sentiment, \\\n",
    "                         product_category \\\n",
    "                FROM {}.{} \\\n",
    "                WHERE product_category = \\'Digital_Video_Download\\' \\\n",
    "                ORDER BY review_date \\\n",
    "                LIMIT 500'\n",
    "               .format(database_name, sentiment_table_name))\n",
    "df = as_pandas(cursor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[['customer_id', 'product_id', 'product_title', 'star_rating', 'review_date']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customer_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>product_title</th>\n",
       "      <th>star_rating</th>\n",
       "      <th>review_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>48685922</td>\n",
       "      <td>B000ICXQVA</td>\n",
       "      <td>Halloween III: Season of the Witch</td>\n",
       "      <td>2</td>\n",
       "      <td>11234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>45764909</td>\n",
       "      <td>B001II0M9W</td>\n",
       "      <td>Simply Irresistible</td>\n",
       "      <td>3</td>\n",
       "      <td>11485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>45764909</td>\n",
       "      <td>B00QFNLJZQ</td>\n",
       "      <td>A.I. Artificial Intelligence</td>\n",
       "      <td>4</td>\n",
       "      <td>11755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>45764909</td>\n",
       "      <td>B00QFNLJZQ</td>\n",
       "      <td>A.I. Artificial Intelligence</td>\n",
       "      <td>4</td>\n",
       "      <td>11755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>45764909</td>\n",
       "      <td>B009941RI2</td>\n",
       "      <td>Wuthering Heights</td>\n",
       "      <td>5</td>\n",
       "      <td>11781</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  customer_id  product_id                       product_title  star_rating  \\\n",
       "0    48685922  B000ICXQVA  Halloween III: Season of the Witch            2   \n",
       "1    45764909  B001II0M9W                 Simply Irresistible            3   \n",
       "2    45764909  B00QFNLJZQ        A.I. Artificial Intelligence            4   \n",
       "3    45764909  B00QFNLJZQ        A.I. Artificial Intelligence            4   \n",
       "4    45764909  B009941RI2                   Wuthering Heights            5   \n",
       "\n",
       "   review_date  \n",
       "0        11234  \n",
       "1        11485  \n",
       "2        11755  \n",
       "3        11755  \n",
       "4        11781  "
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(500, 5)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "customers\n",
      " 0.00     1.0\n",
      "0.01     1.0\n",
      "0.02     1.0\n",
      "0.03     1.0\n",
      "0.04     1.0\n",
      "0.05     1.0\n",
      "0.10     1.0\n",
      "0.25     1.0\n",
      "0.50     1.0\n",
      "0.75     2.0\n",
      "0.90     2.0\n",
      "0.95     3.0\n",
      "0.96     3.0\n",
      "0.97     4.0\n",
      "0.98     5.0\n",
      "0.99     7.0\n",
      "1.00    13.0\n",
      "Name: customer_id, dtype: float64\n",
      "products\n",
      " 0.00     1.00\n",
      "0.01     1.00\n",
      "0.02     1.00\n",
      "0.03     1.00\n",
      "0.04     1.00\n",
      "0.05     1.00\n",
      "0.10     1.00\n",
      "0.25     1.00\n",
      "0.50     1.00\n",
      "0.75     2.00\n",
      "0.90     2.00\n",
      "0.95     4.00\n",
      "0.96     4.00\n",
      "0.97     4.00\n",
      "0.98     6.00\n",
      "0.99     7.92\n",
      "1.00    14.00\n",
      "Name: product_id, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "customers = df['customer_id'].value_counts()\n",
    "products = df['product_id'].value_counts()\n",
    "\n",
    "quantiles = [0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.96, 0.97, 0.98, 0.99, 1]\n",
    "print('customers\\n', customers.quantile(quantiles))\n",
    "print('products\\n', products.quantile(quantiles))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "328\n",
      "309\n",
      "637\n"
     ]
    }
   ],
   "source": [
    "num_customers = customers.count()\n",
    "print(num_customers)\n",
    "num_products = products.count()\n",
    "print(num_products)\n",
    "num_features = num_customers + num_products\n",
    "print(num_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#customers = customers[customers >= 5]\n",
    "#products = products[products >= 10]\n",
    "\n",
    "reduced_df = df.merge(pd.DataFrame({'customer_id': customers.index})).merge(pd.DataFrame({'product_id': products.index}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a sequential index for customers and movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48652051    13\n",
      "47881267     9\n",
      "16336396     8\n",
      "17225887     7\n",
      "45764909     7\n",
      "18147129     5\n",
      "37513072     5\n",
      "53073229     5\n",
      "53070836     5\n",
      "15957545     4\n",
      "16342208     4\n",
      "52824002     4\n",
      "52558011     4\n",
      "17968658     3\n",
      "22119842     3\n",
      "52044147     3\n",
      "46910749     3\n",
      "51302967     3\n",
      "34885907     3\n",
      "52926333     2\n",
      "18436735     2\n",
      "15286599     2\n",
      "9890951      2\n",
      "52843621     2\n",
      "30872364     2\n",
      "44033364     2\n",
      "37040213     2\n",
      "27032484     2\n",
      "43198240     2\n",
      "30288577     2\n",
      "            ..\n",
      "51212387     1\n",
      "43296246     1\n",
      "13106503     1\n",
      "37489005     1\n",
      "24312762     1\n",
      "40115248     1\n",
      "47339197     1\n",
      "40261920     1\n",
      "11606284     1\n",
      "15342724     1\n",
      "52200874     1\n",
      "14992988     1\n",
      "50077390     1\n",
      "16733506     1\n",
      "48206751     1\n",
      "52323087     1\n",
      "43165337     1\n",
      "30023223     1\n",
      "29498914     1\n",
      "47022220     1\n",
      "34905157     1\n",
      "53093634     1\n",
      "48051637     1\n",
      "31041536     1\n",
      "32192325     1\n",
      "50996592     1\n",
      "41484249     1\n",
      "49438424     1\n",
      "52802983     1\n",
      "50845031     1\n",
      "Name: customer_id, Length: 328, dtype: int64\n",
      "B000GT7XXW    14\n",
      "B000MR1V22    10\n",
      "B000NHPCS0     8\n",
      "B000NMW6M0     8\n",
      "B000H4XMPY     7\n",
      "B000IVB07S     7\n",
      "B000MS1XXI     6\n",
      "B000KZ3BWE     6\n",
      "B000LB4DWO     6\n",
      "B000HVHM5S     4\n",
      "B000ICKPQY     4\n",
      "B000ICXQHE     4\n",
      "B000MEBT4A     4\n",
      "B000MF2TB6     4\n",
      "B000MS6NV0     4\n",
      "B000H4QD1O     4\n",
      "B000KI6286     4\n",
      "B000ME5B30     4\n",
      "B000I186FW     4\n",
      "B000HCWZFY     4\n",
      "B000GJPL1S     4\n",
      "B000H06480     4\n",
      "B000HL0EW6     4\n",
      "B000IB1TWE     4\n",
      "B000I01R22     3\n",
      "B000ICNWRI     3\n",
      "B000HSGFWM     3\n",
      "B000MVXB26     3\n",
      "B000NB26NA     3\n",
      "B000I9VZKW     2\n",
      "              ..\n",
      "B000HX6EAU     1\n",
      "B000N4KENG     1\n",
      "B000JULZIM     1\n",
      "B009941RI2     1\n",
      "B004YM33FE     1\n",
      "B000IVN9D6     1\n",
      "B000M28HH4     1\n",
      "B000HAEK40     1\n",
      "B000IVB07I     1\n",
      "B00EVAQSYC     1\n",
      "B00DQJPFIE     1\n",
      "B000KK20BM     1\n",
      "B000I9TWV6     1\n",
      "B000KHZS1Y     1\n",
      "B000JL9GLY     1\n",
      "B000HPF85A     1\n",
      "B000IBUOX4     1\n",
      "B0044WZNCU     1\n",
      "B000NGVP64     1\n",
      "B000KU3TM6     1\n",
      "B000MDHL5M     1\n",
      "B000MW4WIW     1\n",
      "B000I3SZW4     1\n",
      "B000IB2LY4     1\n",
      "B007HJ9GSY     1\n",
      "B000HX3VJM     1\n",
      "B000HZGNJA     1\n",
      "B000M9GE36     1\n",
      "B000GK3BUK     1\n",
      "B000I5P4YE     1\n",
      "Name: product_id, Length: 309, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "customers = reduced_df['customer_id'].value_counts()\n",
    "products = reduced_df['product_id'].value_counts()\n",
    "\n",
    "#customers = df['customer_id'].value_counts()\n",
    "#products = df['product_id'].value_counts()\n",
    "\n",
    "print(customers)\n",
    "print(products)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    customer_id  user\n",
      "0      48652051     0\n",
      "1      47881267     1\n",
      "2      16336396     2\n",
      "3      17225887     3\n",
      "4      45764909     4\n",
      "5      18147129     5\n",
      "6      37513072     6\n",
      "7      53073229     7\n",
      "8      53070836     8\n",
      "9      15957545     9\n",
      "10     16342208    10\n",
      "11     52824002    11\n",
      "12     52558011    12\n",
      "13     17968658    13\n",
      "14     22119842    14\n",
      "15     52044147    15\n",
      "16     46910749    16\n",
      "17     51302967    17\n",
      "18     34885907    18\n",
      "19     52926333    19\n",
      "20     18436735    20\n",
      "21     15286599    21\n",
      "22      9890951    22\n",
      "23     52843621    23\n",
      "24     30872364    24\n",
      "25     44033364    25\n",
      "26     37040213    26\n",
      "27     27032484    27\n",
      "28     43198240    28\n",
      "29     30288577    29\n",
      "..          ...   ...\n",
      "298    51212387   298\n",
      "299    43296246   299\n",
      "300    13106503   300\n",
      "301    37489005   301\n",
      "302    24312762   302\n",
      "303    40115248   303\n",
      "304    47339197   304\n",
      "305    40261920   305\n",
      "306    11606284   306\n",
      "307    15342724   307\n",
      "308    52200874   308\n",
      "309    14992988   309\n",
      "310    50077390   310\n",
      "311    16733506   311\n",
      "312    48206751   312\n",
      "313    52323087   313\n",
      "314    43165337   314\n",
      "315    30023223   315\n",
      "316    29498914   316\n",
      "317    47022220   317\n",
      "318    34905157   318\n",
      "319    53093634   319\n",
      "320    48051637   320\n",
      "321    31041536   321\n",
      "322    32192325   322\n",
      "323    50996592   323\n",
      "324    41484249   324\n",
      "325    49438424   325\n",
      "326    52802983   326\n",
      "327    50845031   327\n",
      "\n",
      "[328 rows x 2 columns]\n",
      "     product_id  item\n",
      "0    B000GT7XXW   328\n",
      "1    B000MR1V22   329\n",
      "2    B000NHPCS0   330\n",
      "3    B000NMW6M0   331\n",
      "4    B000H4XMPY   332\n",
      "5    B000IVB07S   333\n",
      "6    B000MS1XXI   334\n",
      "7    B000KZ3BWE   335\n",
      "8    B000LB4DWO   336\n",
      "9    B000HVHM5S   337\n",
      "10   B000ICKPQY   338\n",
      "11   B000ICXQHE   339\n",
      "12   B000MEBT4A   340\n",
      "13   B000MF2TB6   341\n",
      "14   B000MS6NV0   342\n",
      "15   B000H4QD1O   343\n",
      "16   B000KI6286   344\n",
      "17   B000ME5B30   345\n",
      "18   B000I186FW   346\n",
      "19   B000HCWZFY   347\n",
      "20   B000GJPL1S   348\n",
      "21   B000H06480   349\n",
      "22   B000HL0EW6   350\n",
      "23   B000IB1TWE   351\n",
      "24   B000I01R22   352\n",
      "25   B000ICNWRI   353\n",
      "26   B000HSGFWM   354\n",
      "27   B000MVXB26   355\n",
      "28   B000NB26NA   356\n",
      "29   B000I9VZKW   357\n",
      "..          ...   ...\n",
      "279  B000HX6EAU   607\n",
      "280  B000N4KENG   608\n",
      "281  B000JULZIM   609\n",
      "282  B009941RI2   610\n",
      "283  B004YM33FE   611\n",
      "284  B000IVN9D6   612\n",
      "285  B000M28HH4   613\n",
      "286  B000HAEK40   614\n",
      "287  B000IVB07I   615\n",
      "288  B00EVAQSYC   616\n",
      "289  B00DQJPFIE   617\n",
      "290  B000KK20BM   618\n",
      "291  B000I9TWV6   619\n",
      "292  B000KHZS1Y   620\n",
      "293  B000JL9GLY   621\n",
      "294  B000HPF85A   622\n",
      "295  B000IBUOX4   623\n",
      "296  B0044WZNCU   624\n",
      "297  B000NGVP64   625\n",
      "298  B000KU3TM6   626\n",
      "299  B000MDHL5M   627\n",
      "300  B000MW4WIW   628\n",
      "301  B000I3SZW4   629\n",
      "302  B000IB2LY4   630\n",
      "303  B007HJ9GSY   631\n",
      "304  B000HX3VJM   632\n",
      "305  B000HZGNJA   633\n",
      "306  B000M9GE36   634\n",
      "307  B000GK3BUK   635\n",
      "308  B000I5P4YE   636\n",
      "\n",
      "[309 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "customer_index = pd.DataFrame({'customer_id': customers.index, \n",
    "                               'user': np.arange(customers.shape[0])})\n",
    "print(customer_index)\n",
    "\n",
    "product_index = pd.DataFrame({'product_id': products.index, \n",
    "                              'item': np.arange(products.shape[0]) + customer_index.shape[0]})\n",
    "print(product_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Train\n",
    "\n",
    "- Create a [SageMaker Python SDK](https://github.com/aws/sagemaker-python-sdk) estimator to run a training jobs and specify:\n",
    "  - Algorithm container image\n",
    "  - IAM role\n",
    "  - Hardware setup\n",
    "  - S3 output location\n",
    "  - Algorithm hyperparameters\n",
    "    - `feature_dim`: $N + M + 1$ (additional feature is `days_since_first` to capture trend)\n",
    "    - `num_factors`: number of factor dimensions (increasing too much can lead to overfitting)\n",
    "    - `epochs`: number of full passes through the dataset\n",
    "- `.fit()` points to training and test data in S3 and begins the training job\n",
    "\n",
    "**Note**: For AWS accounts registered in conjunction with a workshop, default instance limits may prevent the use of `ml.c5.2xlarge` (and other equally powerful instances), and may require a lower value for `train_instance_count` depending on the instance type chosen. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "role = sagemaker.get_execution_role()\n",
    "\n",
    "fm_prefix = 'factorization-machines'\n",
    "\n",
    "base_job_name = 'factorization-machines'\n",
    "\n",
    "fm = sagemaker.estimator.Estimator(\n",
    "    sagemaker.amazon.amazon_estimator.get_image_uri(boto3.Session().region_name, 'factorization-machines', 'latest'),\n",
    "    role, \n",
    "    train_instance_count=1,\n",
    "    train_instance_type='ml.c5.xlarge',\n",
    "    output_path='s3://{}/{}/output'.format(bucket, fm_prefix),\n",
    "    base_job_name=base_job_name,\n",
    "    sagemaker_session=sagemaker_session)\n",
    "\n",
    "fm.set_hyperparameters(\n",
    "    feature_dim=customer_index.shape[0] + product_index.shape[0] + 1,\n",
    "    predictor_type='regressor',\n",
    "    mini_batch_size=1000,\n",
    "    num_factors=256,\n",
    "    epochs=3)\n",
    "\n",
    "fm.fit({'train': sagemaker.s3_input('s3://{}/{}/train/'.format(bucket, fm_prefix), distribution='ShardedByS3Key'), \n",
    "        'test': sagemaker.s3_input('s3://{}/{}/test/'.format(bucket, fm_prefix), distribution='FullyReplicated')},\n",
    "        wait=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_job_name:  factorization-machines-2020-02-13-02-38-03-444\n"
     ]
    }
   ],
   "source": [
    "training_job_name = fm.latest_training_job.name\n",
    "print('training_job_name:  {}'.format(training_job_name))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After some time, or in a separate Python notebook, we can attach to the running job using the `training_job_name`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-13 02:38:03 Starting - Starting the training job...\n",
      "2020-02-13 02:38:05 Starting - Launching requested ML instances.........\n",
      "2020-02-13 02:39:39 Starting - Preparing the instances for training...\n",
      "2020-02-13 02:40:31 Downloading - Downloading input data......\n",
      "2020-02-13 02:41:31 Training - Training image download completed. Training in progress.\n",
      "2020-02-13 02:41:31 Uploading - Uploading generated training model\u001b[34mDocker entrypoint called with argument(s): train\u001b[0m\n",
      "\u001b[34m/opt/amazon/lib/python2.7/site-packages/pandas/util/nosetester.py:13: DeprecationWarning: Importing from numpy.testing.nosetester is deprecated, import from numpy.testing instead.\n",
      "  from numpy.testing import nosetester\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] Reading default configuration from /opt/amazon/lib/python2.7/site-packages/algorithm/resources/default-conf.json: {u'factors_lr': u'0.0001', u'linear_init_sigma': u'0.01', u'epochs': 1, u'_wd': u'1.0', u'_num_kv_servers': u'auto', u'use_bias': u'true', u'factors_init_sigma': u'0.001', u'_log_level': u'info', u'bias_init_method': u'normal', u'linear_init_method': u'normal', u'linear_lr': u'0.001', u'factors_init_method': u'normal', u'_tuning_objective_metric': u'', u'bias_wd': u'0.01', u'use_linear': u'true', u'bias_lr': u'0.1', u'mini_batch_size': u'1000', u'_use_full_symbolic': u'true', u'batch_metrics_publish_interval': u'500', u'bias_init_sigma': u'0.01', u'_num_gpus': u'auto', u'_data_format': u'record', u'factors_wd': u'0.00001', u'linear_wd': u'0.001', u'_kvstore': u'auto', u'_learning_rate': u'1.0', u'_optimizer': u'adam'}\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] Reading provided configuration from /opt/ml/input/config/hyperparameters.json: {u'epochs': u'3', u'feature_dim': u'638', u'mini_batch_size': u'1000', u'predictor_type': u'regressor', u'num_factors': u'256'}\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] Final configuration: {u'factors_lr': u'0.0001', u'linear_init_sigma': u'0.01', u'epochs': u'3', u'feature_dim': u'638', u'num_factors': u'256', u'_wd': u'1.0', u'_num_kv_servers': u'auto', u'use_bias': u'true', u'factors_init_sigma': u'0.001', u'_log_level': u'info', u'bias_init_method': u'normal', u'linear_init_method': u'normal', u'linear_lr': u'0.001', u'factors_init_method': u'normal', u'_tuning_objective_metric': u'', u'bias_wd': u'0.01', u'use_linear': u'true', u'bias_lr': u'0.1', u'mini_batch_size': u'1000', u'_use_full_symbolic': u'true', u'batch_metrics_publish_interval': u'500', u'predictor_type': u'regressor', u'bias_init_sigma': u'0.01', u'_num_gpus': u'auto', u'_data_format': u'record', u'factors_wd': u'0.00001', u'linear_wd': u'0.001', u'_kvstore': u'auto', u'_learning_rate': u'1.0', u'_optimizer': u'adam'}\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 WARNING 140211299915584] Loggers have already been setup.\u001b[0m\n",
      "\u001b[34mProcess 1 is a worker.\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] Using default worker.\u001b[0m\n",
      "\u001b[34m[2020-02-13 02:41:28.527] [tensorio] [warning] TensorIO is already initialized; ignoring the initialization routine.\u001b[0m\n",
      "\u001b[34m[2020-02-13 02:41:28.528] [tensorio] [info] epoch_stats={\"data_pipeline\": \"/opt/ml/input/data/train\", \"epoch\": 0, \"duration\": 2, \"num_examples\": 1, \"num_bytes\": 20992}\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] nvidia-smi took: 0.0251371860504 secs to identify 0 gpus\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] Number of GPUs being used: 0\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] [Sparse network] Building a sparse network.\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] Create Store: local\u001b[0m\n",
      "\u001b[34m#metrics {\"Metrics\": {\"initialize.time\": {\"count\": 1, \"max\": 33.90097618103027, \"sum\": 33.90097618103027, \"min\": 33.90097618103027}}, \"EndTime\": 1581561688.561669, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1581561688.523362}\n",
      "\u001b[0m\n",
      "\u001b[34m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 0, \"sum\": 0.0, \"min\": 0}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 0, \"sum\": 0.0, \"min\": 0}, \"Total Batches Seen\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}, \"Total Records Seen\": {\"count\": 1, \"max\": 328, \"sum\": 328.0, \"min\": 328}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 328, \"sum\": 328.0, \"min\": 328}, \"Reset Count\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}}, \"EndTime\": 1581561688.561849, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"init_train_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1581561688.561806}\n",
      "\u001b[0m\n",
      "\u001b[34m[02:41:28] /opt/brazil-pkg-cache/packages/AIAlgorithmsMXNet/AIAlgorithmsMXNet-1.1.x.202222.0/AL2012/generic-flavor/src/src/kvstore/./kvstore_local.h:280: Warning: non-default weights detected during kvstore pull. This call has been ignored. Please make sure to use row_sparse_pull with row_ids.\u001b[0m\n",
      "\u001b[34m[02:41:28] /opt/brazil-pkg-cache/packages/AIAlgorithmsMXNet/AIAlgorithmsMXNet-1.1.x.202222.0/AL2012/generic-flavor/src/src/kvstore/./kvstore_local.h:280: Warning: non-default weights detected during kvstore pull. This call has been ignored. Please make sure to use row_sparse_pull with row_ids.\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=0, batch=0 train rmse <loss>=2.2251415422\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=0, batch=0 train mse <loss>=4.95125488281\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=0, batch=0 train absolute_loss <loss>=1.16560998535\u001b[0m\n",
      "\u001b[34m[2020-02-13 02:41:28.656] [tensorio] [info] epoch_stats={\"data_pipeline\": \"/opt/ml/input/data/train\", \"epoch\": 2, \"duration\": 71, \"num_examples\": 1, \"num_bytes\": 20992}\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=0, train rmse <loss>=2.2251415422\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=0, train mse <loss>=4.95125488281\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=0, train absolute_loss <loss>=1.16560998535\u001b[0m\n",
      "\u001b[34m#metrics {\"Metrics\": {\"epochs\": {\"count\": 1, \"max\": 3, \"sum\": 3.0, \"min\": 3}, \"update.time\": {\"count\": 1, \"max\": 95.15595436096191, \"sum\": 95.15595436096191, \"min\": 95.15595436096191}}, \"EndTime\": 1581561688.657183, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1581561688.561738}\n",
      "\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #progress_metric: host=algo-1, completed 33 % of epochs\u001b[0m\n",
      "\u001b[34m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 328, \"sum\": 328.0, \"min\": 328}, \"Total Batches Seen\": {\"count\": 1, \"max\": 2, \"sum\": 2.0, \"min\": 2}, \"Total Records Seen\": {\"count\": 1, \"max\": 656, \"sum\": 656.0, \"min\": 656}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 328, \"sum\": 328.0, \"min\": 328}, \"Reset Count\": {\"count\": 1, \"max\": 2, \"sum\": 2.0, \"min\": 2}}, \"EndTime\": 1581561688.657425, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 0}, \"StartTime\": 1581561688.561992}\n",
      "\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #throughput_metric: host=algo-1, train throughput=3433.54375877 records/second\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] Iter[1] Batch [0]#011Speed: 37389053.31 samples/sec\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=1, batch=0 train rmse <loss>=2.17579356482\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=1, batch=0 train mse <loss>=4.73407763672\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=1, batch=0 train absolute_loss <loss>=1.20001013184\u001b[0m\n",
      "\u001b[34m[2020-02-13 02:41:28.669] [tensorio] [info] epoch_stats={\"data_pipeline\": \"/opt/ml/input/data/train\", \"epoch\": 4, \"duration\": 10, \"num_examples\": 1, \"num_bytes\": 20992}\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=1, train rmse <loss>=2.17579356482\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=1, train mse <loss>=4.73407763672\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=1, train absolute_loss <loss>=1.20001013184\u001b[0m\n",
      "\u001b[34m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 12.238025665283203, \"sum\": 12.238025665283203, \"min\": 12.238025665283203}}, \"EndTime\": 1581561688.669928, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1581561688.657263}\n",
      "\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #progress_metric: host=algo-1, completed 66 % of epochs\u001b[0m\n",
      "\u001b[34m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 328, \"sum\": 328.0, \"min\": 328}, \"Total Batches Seen\": {\"count\": 1, \"max\": 3, \"sum\": 3.0, \"min\": 3}, \"Total Records Seen\": {\"count\": 1, \"max\": 984, \"sum\": 984.0, \"min\": 984}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 328, \"sum\": 328.0, \"min\": 328}, \"Reset Count\": {\"count\": 1, \"max\": 3, \"sum\": 3.0, \"min\": 3}}, \"EndTime\": 1581561688.670069, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 1}, \"StartTime\": 1581561688.657665}\n",
      "\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #throughput_metric: host=algo-1, train throughput=26246.4078144 records/second\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] Iter[2] Batch [0]#011Speed: 43110471.57 samples/sec\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=2, batch=0 train rmse <loss>=2.13015436464\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=2, batch=0 train mse <loss>=4.53755761719\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=2, batch=0 train absolute_loss <loss>=1.23428991699\u001b[0m\n",
      "\u001b[34m[2020-02-13 02:41:28.681] [tensorio] [info] epoch_stats={\"data_pipeline\": \"/opt/ml/input/data/train\", \"epoch\": 6, \"duration\": 10, \"num_examples\": 1, \"num_bytes\": 20992}\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=2, train rmse <loss>=2.13015436464\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=2, train mse <loss>=4.53755761719\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, epoch=2, train absolute_loss <loss>=1.23428991699\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, train rmse <loss>=2.13015436464\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, train mse <loss>=4.53755761719\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, train absolute_loss <loss>=1.23428991699\u001b[0m\n",
      "\u001b[34m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 11.85297966003418, \"sum\": 11.85297966003418, \"min\": 11.85297966003418}}, \"EndTime\": 1581561688.682093, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1581561688.669979}\n",
      "\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #progress_metric: host=algo-1, completed 100 % of epochs\u001b[0m\n",
      "\u001b[34m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 328, \"sum\": 328.0, \"min\": 328}, \"Total Batches Seen\": {\"count\": 1, \"max\": 4, \"sum\": 4.0, \"min\": 4}, \"Total Records Seen\": {\"count\": 1, \"max\": 1312, \"sum\": 1312.0, \"min\": 1312}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 328, \"sum\": 328.0, \"min\": 328}, \"Reset Count\": {\"count\": 1, \"max\": 4, \"sum\": 4.0, \"min\": 4}}, \"EndTime\": 1581561688.682226, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 2}, \"StartTime\": 1581561688.670221}\n",
      "\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #throughput_metric: host=algo-1, train throughput=27145.4560379 records/second\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 WARNING 140211299915584] wait_for_all_workers will not sync workers since the kv store is not running distributed\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] Pulling entire model from kvstore to finalize\u001b[0m\n",
      "\u001b[34m#metrics {\"Metrics\": {\"finalize.time\": {\"count\": 1, \"max\": 1.3301372528076172, \"sum\": 1.3301372528076172, \"min\": 1.3301372528076172}}, \"EndTime\": 1581561688.683737, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1581561688.682144}\n",
      "\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] Saved checkpoint to \"/tmp/tmpJXxn6h/state-0001.params\"\u001b[0m\n",
      "\u001b[34m[2020-02-13 02:41:28.689] [tensorio] [info] epoch_stats={\"data_pipeline\": \"/opt/ml/input/data/test\", \"epoch\": 0, \"duration\": 162, \"num_examples\": 1, \"num_bytes\": 5376}\u001b[0m\n",
      "\u001b[34m[2020-02-13 02:41:28.693] [tensorio] [info] epoch_stats={\"data_pipeline\": \"/opt/ml/input/data/test\", \"epoch\": 1, \"duration\": 4, \"num_examples\": 1, \"num_bytes\": 5376}\u001b[0m\n",
      "\u001b[34m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 84, \"sum\": 84.0, \"min\": 84}, \"Total Batches Seen\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}, \"Total Records Seen\": {\"count\": 1, \"max\": 84, \"sum\": 84.0, \"min\": 84}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 84, \"sum\": 84.0, \"min\": 84}, \"Reset Count\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}}, \"EndTime\": 1581561688.693639, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"test_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1581561688.689203}\n",
      "\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #test_score (algo-1) : ('rmse', 3.739623229824318)\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #test_score (algo-1) : ('mse', 13.984781901041666)\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #test_score (algo-1) : ('absolute_loss', 3.447500683012463)\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, test rmse <loss>=3.73962322982\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, test mse <loss>=13.984781901\u001b[0m\n",
      "\u001b[34m[02/13/2020 02:41:28 INFO 140211299915584] #quality_metric: host=algo-1, test absolute_loss <loss>=3.44750068301\u001b[0m\n",
      "\u001b[34m#metrics {\"Metrics\": {\"totaltime\": {\"count\": 1, \"max\": 213.6859893798828, \"sum\": 213.6859893798828, \"min\": 213.6859893798828}, \"setuptime\": {\"count\": 1, \"max\": 38.03586959838867, \"sum\": 38.03586959838867, \"min\": 38.03586959838867}}, \"EndTime\": 1581561688.694674, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1581561688.683785}\n",
      "\u001b[0m\n",
      "\n",
      "2020-02-13 02:41:38 Completed - Training job completed\n",
      "Training seconds: 67\n",
      "Billable seconds: 67\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.estimator import Estimator\n",
    "\n",
    "fm = Estimator.attach(training_job_name=training_job_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Host\n",
    "\n",
    "Deploy trained model to a real-time production endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------!"
     ]
    }
   ],
   "source": [
    "fm_predictor = fm.deploy(instance_type='ml.m4.xlarge', initial_instance_count=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Setup predictor to serialize in-memory data for invocation requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "def fm_serializer(df):\n",
    "    feature_dim = customer_index.shape[0] + product_index.shape[0] + 1\n",
    "    js = {'instances': []}\n",
    "    for index, data in df.iterrows():\n",
    "        js['instances'].append({'data': {'features': {'values': [1, 1, data['days_since_first']],\n",
    "                                                      'keys': [data['user'], data['item'], feature_dim - 1],\n",
    "                                                      'shape': [feature_dim]}}})\n",
    "    return json.dumps(js)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.predictor import json_deserializer\n",
    "\n",
    "fm_predictor.content_type = 'application/json'\n",
    "fm_predictor.serializer = fm_serializer\n",
    "fm_predictor.deserializer = json_deserializer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Real-time prediction for what a single user would rate an item**\n",
    "\n",
    "1. Pick a customer-movie pair from the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'test_df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-78-29726e40b4fa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtest_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'test_df' is not defined"
     ]
    }
   ],
   "source": [
    "test_df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. Pull out a single customer-product pair that we like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_customer = test_df.iloc[[0]]\n",
    "test_customer = test_customer[['customer_id', 'user', 'item', 'days_since_first']]\n",
    "\n",
    "test_customer.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. Pass `test_customer` to predictor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fm_predictor.predict(test_customer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1 entries, 0 to 0\n",
      "Data columns (total 4 columns):\n",
      "customer_id         1 non-null object\n",
      "user                1 non-null int64\n",
      "item                1 non-null int64\n",
      "days_since_first    1 non-null int64\n",
      "dtypes: int64(3), object(1)\n",
      "memory usage: 40.0+ bytes\n"
     ]
    }
   ],
   "source": [
    "#fake_customer = test_customer \n",
    "desired_customer_id = '45764909' # test_customer['customer_id']\n",
    "desired_user_id = 4 # int(test_customer['user'])\n",
    "desired_item_id = 508 # int(test_customer['item'])\n",
    "desired_review_days = 0\n",
    "\n",
    "type(desired_review_days)\n",
    "fake_customer_data = {'customer_id': desired_customer_id, \n",
    "                      'user' : desired_user_id, \n",
    "                      'item' : desired_item_id, \n",
    "                      'days_since_first' : desired_review_days}\n",
    "fake_customer = pd.DataFrame(fake_customer_data, index=[0])\n",
    "#fake_customer['customer_id'] = desired_customer_id\n",
    "#fake_customer['user'] = desired_user_id\n",
    "#fake_customer['item'] = desired_item_id\n",
    "#fake_customer['days_since_first'] = desired_review_days\n",
    "\n",
    "fake_customer.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1 entries, 0 to 0\n",
      "Data columns (total 4 columns):\n",
      "customer_id         1 non-null object\n",
      "user                1 non-null int64\n",
      "item                1 non-null int64\n",
      "days_since_first    1 non-null int64\n",
      "dtypes: int64(3), object(1)\n",
      "memory usage: 40.0+ bytes\n"
     ]
    }
   ],
   "source": [
    "#fake_customer = test_customer \n",
    "desired_customer_id = '10653514' # test_customer['customer_id']\n",
    "desired_user_id = 240 # int(test_customer['user'])\n",
    "desired_item_id = 451 # int(test_customer['item'])\n",
    "desired_review_days = 0\n",
    "\n",
    "type(desired_review_days)\n",
    "fake_customer_data = {'customer_id': desired_customer_id, \n",
    "                      'user' : desired_user_id, \n",
    "                      'item' : desired_item_id, \n",
    "                      'days_since_first' : desired_review_days}\n",
    "fake_customer = pd.DataFrame(fake_customer_data, index=[0])\n",
    "#fake_customer['customer_id'] = desired_customer_id\n",
    "#fake_customer['user'] = desired_user_id\n",
    "#fake_customer['item'] = desired_item_id\n",
    "#fake_customer['days_since_first'] = desired_review_days\n",
    "\n",
    "fake_customer.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'predictions': [{'score': 0.30700334906578064}]}"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fm_predictor.predict(fake_customer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean-up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fm_predictor.delete_endpoint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
